from collections import namedtuple

text = 'foo = 23 + 42 * 10'
import re

#?P<TOKENNAME> 用于给一个模式命名
NAME = r'(?P<NAME>[a-zA-Z_][A-Za-z0-9]*)'
NUM = r'(?P<NUM>\d+)'
PLUS = r'(?P<PLUS>\+)'
TIMES = r'(?P<TIMES>\*)'
EQ = r'(?P<EQ>=)'
WS = r'(?P<WS>\s+)'

master_pat = re.compile('|'.join([NAME,NUM,PLUS,TIMES,EQ,WS]))

'''
>>> scanner = master_pat.scanner('foo = 42')
>>> scanner.match()
<_sre.SRE_Match object at 0x100677738>
>>> _.lastgroup, _.group()
('NAME', 'foo')
>>> scanner.match()
<_sre.SRE_Match object at 0x100677738>
>>> _.lastgroup, _.group()
('WS', ' ')
>>> scanner.match()
<_sre.SRE_Match object at 0x100677738>
>>> _.lastgroup, _.group()
('EQ', '=')
>>> scanner.match()
<_sre.SRE_Match object at 0x100677738>
>>> _.lastgroup, _.group()
('WS', ' ')
>>> scanner.match()
<_sre.SRE_Match object at 0x100677738>
>>> _.lastgroup, _.group()
('NUM', '42')
'''

def generate_tokens(pat,text):
    Token = namedtuple('Token', ['type', 'value'])
    scanner = pat.scanner(text)
    for m in iter(scanner.match,None):  # 迭代器函数iter()
        yield Token(m.lastgroup,m.group())

if __name__ == '__main__':
    for tok in generate_tokens(master_pat,'foo = 42'):
        print(tok)

if __name__ == '__main__':
    PRINT = r'(?P<PRINT>print)'
    NAME = r'(?P<NAME>[A-Za-z][a-zA-Z0-9_]*)'
    master_pat = re.compile('|'.join([PRINT,NAME]))
    for tok in generate_tokens(master_pat,'printer11'):
        print(tok)